
import sys
sys.path.append('D:\wordSpace\langchainQA')
import re
import io

import requests
from uuid import uuid4
from bs4 import BeautifulSoup
from PIL import Image

from app.utils.oss2_upload import Oss2UploadUtil

oss=Oss2UploadUtil()

def get_html(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    response.encoding = 'utf-8'
    return response.text

def match_img_type(url):

    # Use regex to find the part after "wx_fmt="
    match = re.search(r'wx_fmt=(\w+)', url)

    if match:
        image_format = match.group(1)
        print(image_format)
    return image_format
def html_to_markdown(html):
    soup = BeautifulSoup(html, 'html.parser')
    img_tags = soup.find_all('img',{'data-src': True,'class':"wxw-img"})
    js_pc_qr_code=soup.find(id="js_pc_qr_code")
    xhr_scripts = soup.find_all(lambda tag: tag.name == 'script' and 'xhr' in tag.text)
    for item in xhr_scripts:
        # print(item)
        item.clear()
    # print(len(img_tags))
    js_pc_qr_code.clear()
    # 3. 存储图片的本地路径字典
    image_dict = {}
    # for item in scprit_tags:
    #     print(item)
    #     item.clear()

    # 4. 下载图片并保存到本地
    for img_tag in img_tags:
        
        img_url = img_tag.get('data-src',)
        # print(img_url,match_img_type(img_url))
        if img_url:
            img_type=match_img_type(img_url)
            # 构建本地文件名
            # img_filename = os.path.join(os.getcwd(), )
            img_filename=str(uuid4().hex)
            # 下载图片（这里使用了requests库）
            try:
                with requests.get(img_url) as img_response:
                    img_response.raise_for_status()
                    file_path=f"{img_filename}.{img_type}"
                    res= oss.upload('img',fileName=file_path,fileobj=img_response)
                    img_tag['src']=res
                image_dict[img_url] = file_path
            except requests.exceptions.RequestException as e:
                print(f"下载图片失败: {str(e)}")
    new_html_content =str(soup)
    return new_html_content

if __name__ == "__main__":
    url = "https://mp.weixin.qq.com/s/liM73uUyYEx8HgnaVXviUQ"
    html = get_html(url)
    html2 = html_to_markdown(html)
    # print(html2)
    with open('output3.html', 'w', encoding='utf-8') as f:
        print(1111)
        f.write(html2)
    with open('output3.html', 'rb') as file:
        file_data = file.read()
    res= oss.upload('img',fileName="test.html",fileobj=file_data)
    print(res)